package Basic

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object Spark_WordCount {
  def main(args: Array[String]): Unit = {
    //todo 建立和spark框架的连接
    val sparkConf = new SparkConf().setMaster("local").setAppName("WordCount")
    val sc = new SparkContext(sparkConf)
    sc.setLogLevel("ERROR")
    //todo 执行业务操作

    //1、读取文件,读取一行一行的数据
    //hello world
    var lines: RDD[String] = sc.textFile("dataset/WordCount")

    //2、将一行数据进行拆分
    //"hello world" =>hello,world,hello,world
    //扁平化:将整体拆分成个体
    val word: RDD[String] = lines.flatMap(_.split(" "))

    //3、将数据进行分组
    //(hello,hello,hello)
    val wordGroup: RDD[(String, Iterable[String])] = word.groupBy(word => word)

    //4、对分组后的数据进行转换
    //(hello,3)
    val wordToCount = wordGroup.map {
      case (word, list) => {
        (word,list.size)
      }
    }

    //将结果转换到控制台打印出来
    val array: Array[(String,Int)] = wordToCount.collect()
    array.foreach(println)

    //todo 关闭连接
    sc.stop()
  }
}

